{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Brainstorming and Focus Group Quantitative Experimentation 1: **General US population** under **action correction** + **divergence intervention**\n",
    "\n",
    "Can we use TinyTroupe to brainstorm product ideas?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "!!!!\n",
      "DISCLAIMER: TinyTroupe relies on Artificial Intelligence (AI) models to generate content. \n",
      "The AI models are not perfect and may produce inappropriate or inacurate results. \n",
      "For any serious or consequential use, please review the generated content before using it.\n",
      "!!!!\n",
      "\n",
      "Looking for default config on: c:\\Users\\pdasilva\\AppData\\Local\\anaconda3\\envs\\py310\\lib\\site-packages\\tinytroupe\\utils\\..\\config.ini\n",
      "Found custom config on: c:\\Users\\pdasilva\\OneDrive - Microsoft\\TinyTroupe (shared)\\Paper artifacts\\Working examples\\config.ini\n",
      "\n",
      "=================================\n",
      "Current TinyTroupe configuration \n",
      "=================================\n",
      "[OpenAI]\n",
      "api_type = openai\n",
      "azure_api_version = 2024-08-01-preview\n",
      "model = gpt-4o-mini\n",
      "reasoning_model = o3-mini\n",
      "embedding_model = text-embedding-3-small\n",
      "max_tokens = 16000\n",
      "temperature = 1.2\n",
      "freq_penalty = 0.0\n",
      "presence_penalty = 0.0\n",
      "timeout = 60\n",
      "max_attempts = 5\n",
      "waiting_time = 0\n",
      "exponential_backoff_factor = 5\n",
      "reasoning_effort = high\n",
      "cache_api_calls = False\n",
      "cache_file_name = openai_api_cache.pickle\n",
      "max_content_display_length = 1024\n",
      "azure_embedding_model_api_version = 2023-05-15\n",
      "\n",
      "[Simulation]\n",
      "parallel_agent_generation = True\n",
      "parallel_agent_actions = True\n",
      "rai_harmful_content_prevention = True\n",
      "rai_copyright_infringement_prevention = True\n",
      "\n",
      "[ActionGenerator]\n",
      "max_attempts = 2\n",
      "enable_quality_checks = False\n",
      "enable_regeneration = True\n",
      "enable_direct_correction = False\n",
      "enable_quality_check_for_persona_adherence = True\n",
      "enable_quality_check_for_selfconsistency = False\n",
      "enable_quality_check_for_fluency = False\n",
      "enable_quality_check_for_suitability = False\n",
      "enable_quality_check_for_similarity = False\n",
      "continue_on_failure = True\n",
      "quality_threshold = 5\n",
      "\n",
      "[Logging]\n",
      "loglevel = WARNING\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "\n",
    "from pprint import pprint\n",
    "\n",
    "from tinytroupe.agent import TinyPerson\n",
    "from tinytroupe.environment import TinyWorld\n",
    "from tinytroupe.experimentation import InPlaceExperimentRunner\n",
    "from tinytroupe.steering import Intervention\n",
    "from tinytroupe.examples import *\n",
    "from tinytroupe.validation import propositions\n",
    "from tinytroupe.extraction import ResultsExtractor\n",
    "from tinytroupe.utils.parallel import parallel_map_dict, parallel_map_cross\n",
    "from tinytroupe.validation import hard_persona_adherence, persona_adherence, self_consistency, fluency, task_completion, divergence\n",
    "\n",
    "# specific utilities\n",
    "from common_utils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_mode = True  # set to True to run the full mode with all agents and tasks\n",
    "\n",
    "# avoid displaying the communication, to make the output cleaner for eval\n",
    "TinyPerson.communication_display = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if full_mode:\n",
    "    repetitions_per_task = 2\n",
    "    simulation_steps = 10\n",
    "    qty_agents = 16\n",
    "    qty_proposals = 4\n",
    "\n",
    "else:\n",
    "    repetitions_per_task = 1\n",
    "    simulation_steps = 10\n",
    "    qty_agents = 12\n",
    "    qty_proposals = 1\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-06-05 13:04:39,116 - tinytroupe - WARNING - Configuration file './brainstorming_and_focus_group_quantitative_experimentation_1c.json' exists and was loaded successfully. If you are trying to fully rerun the experiments, delete it first.\n"
     ]
    }
   ],
   "source": [
    "experiment_runner = InPlaceExperimentRunner(\"./brainstorming_and_focus_group_quantitative_experimentation_1c.json\")\n",
    "\n",
    "experiment_runner.add_experiment(\"Control\")\n",
    "experiment_runner.add_experiment(\"Treatment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "experiment_runner.activate_next_experiment()\n",
    "\n",
    "#experiment_runner.fix_active_experiment(\"Control\")\n",
    "#experiment_runner.fix_active_experiment(\"Treatment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Running experiment None\n"
     ]
    }
   ],
   "source": [
    "print(f\"Running experiment {experiment_runner.get_active_experiment()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agents and populations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "people = []\n",
    "if not experiment_runner.has_finished_all_experiments():\n",
    "    # load agents\n",
    "    people = TinyPerson.load_specifications_from_folder(\"./population/usa_general\")\n",
    "\n",
    "    # filter to make it go faster?\n",
    "    if not full_mode:\n",
    "        people = people[:qty_agents]\n",
    "\n",
    "    # customize and print minibios \n",
    "    for person in people:\n",
    "        ### person.import_fragment(\"./fragments/picky_customer.agent.fragment.json\")\n",
    "        print(person.minibio(extended=False))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(people)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# divide people in several groups of 5\n",
    "people_groups = []\n",
    "for i in range(0, len(people), 5):\n",
    "    people_groups.append(people[i:i+5]\n",
    "    )\n",
    "\n",
    "len(people_groups)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The experiment refers to customers\n",
    "\n",
    "if experiment_runner.get_active_experiment() == \"Control\":\n",
    "    for person in people:\n",
    "        person.action_generator.enable_reasoning_step = False\n",
    "        person.action_generator.enable_quality_checks = False\n",
    "\n",
    "elif experiment_runner.get_active_experiment() == \"Treatment\":    \n",
    "    for person in people:\n",
    "       person.action_generator.enable_reasoning_step = False\n",
    "       person.action_generator.enable_quality_checks = True\n",
    "       person.action_generator.max_attempts = 5\n",
    "       person.action_generator.enable_regeneration = True\n",
    "       person.action_generator.quality_threshold = 5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Proposals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "proposals = [\n",
    "    {\"theme\": \"Food and Nutrition (food itself, consumption, preparation, transportation, storage)\", \n",
    "    \"objective\": \"ideas for new food products, either new foods, food services, food experiences, \"+\\\n",
    "                \"or food preparation tools.\" },\n",
    "    {\"theme\": \"Travel and Tourism (travel, tourism, hospitality, leisure)\",\n",
    "    \"objective\": \"ideas for new travel and tourism services, experiences, or products.\" },\n",
    "    {\"theme\": \"Health and Wellbeing (health, wellness, fitness, beauty)\",\n",
    "    \"objective\": \"ideas for new health and wellbeing services, experiences, or products.\" },\n",
    "    {\"theme\": \"Economics and Finance (economics, finance, business, work)\",\n",
    "    \"objective\": \"ideas for new economic and financial services, experiences, or products.\" },\n",
    "    {\"theme\": \"Technology and Innovation (technology, innovation, science, research)\",\n",
    "    \"objective\": \"ideas for new technology and innovation services, experiences, or products.\" }\n",
    "]\n",
    "\n",
    "if not full_mode:\n",
    "    proposals = proposals[:qty_proposals]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Auxiliary functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def brainstorming_battery(agents, proposals, interventions, agent_propositions, environment_propositions, \n",
    "                          repetitions = 5, simulation_steps=10): \n",
    "    \n",
    "    agent_propositions_scores = {}\n",
    "    environment_propositions_scores = {}\n",
    "\n",
    "    experiments_count = 0\n",
    "    total_expected_experiments = len(proposals) * repetitions #* len(agents)\n",
    "\n",
    "    # TODO remove?\n",
    "    #\n",
    "    # Add intervention to prevent agents from being too quiet.\n",
    "    #for agent in agents:\n",
    "    #    intervention = \\\n",
    "    #        Intervention(agent)\\\n",
    "    #            .set_propositional_precondition(propositions.quiet_recently)\\\n",
    "    #            .set_effect(lambda target: target.think(\"\"\"\n",
    "    #                                                    I will say something now, I've been too quiet for a while. If I am uncomfortable, \n",
    "    #                                                    or can't think of a proper response,\n",
    "    #                                                    I can always say something like \"I don't want to talk about this\",\n",
    "    #                                                    or propose another topic.\n",
    "    #                                                    \"\"\"))\n",
    "    #    interventions.append(intervention)\n",
    "\n",
    "    # loop over proposals and repetitions\n",
    "    for proposal in proposals:\n",
    "\n",
    "        objective = proposal[\"objective\"]\n",
    "        theme = proposal[\"theme\"]\n",
    "\n",
    "        for i in range(repetitions):\n",
    "            print(\"\\n############## STARTING A NEW RESEARCH SESSION #################\")\n",
    "            print(f\"Overall experiment number: {experiments_count+1} / {total_expected_experiments}\")\n",
    "            print(f\"Discussion objective: {objective}\")\n",
    "            print(f\"Trial number: {i+1}\")\n",
    "            print(f\"Agents: {agents}\")\n",
    "\n",
    "            # clear the episodic memory of all agents\n",
    "            for person in agents:\n",
    "                person.clear_episodic_memory()\n",
    "\n",
    "            world = TinyWorld(agents=agents, interventions=interventions)\n",
    "            \n",
    "            # Participants introduce themselves\n",
    "            world.broadcast(f\"\"\"\n",
    "                Hello everyone! Let's start by introducing ourselves, and mentioning problems we face in our daily personal\n",
    "                and professional lives related to the following theme: {theme}\n",
    "                \n",
    "                Please:\n",
    "                  - present yourself and your background;\n",
    "                  - present some key personal problems related to the theme;\n",
    "                  - present some key problems related to the theme that you face in your work;\n",
    "                  - present some key problems related to the theme that you see in your industry as a whole.\n",
    "                  \n",
    "                Don't discuss solutions yet, just the problems you face and see others facing.\n",
    "                \"\"\")\n",
    "            world.run(1)\n",
    "            \n",
    "            # now to the brainstorming session itself\n",
    "            world.broadcast(f\"\"\"\n",
    "                Folks, your mission is to brainstorm {objective}. \n",
    "                Please follow these guidelines:\n",
    "                  - give a unique and informative name to each idea you propose, so that it is easy to refer to it. Say it like \"Idea name: '<name of the idea>'\".;\n",
    "                  - explain why you think it is a good idea, and what problem it solves, and how you feel about it;\n",
    "                  - your ideas should be new complete, self-contained, products or services, not features for other existing products or services;\n",
    "                  - think of creative ideas that would somehow help you in both in your personal and professional lives.\n",
    "                  - create as many different and unique ideas as you can during the brainstorming session. Each idea must be **completely** different from the others \n",
    "                    (either by yourself or by others), and not just a variation of an existing idea.\n",
    "                    and not just a variation of an existing idea.\n",
    "                  - you should criticize each other's ideas, in order to make sure they are as\n",
    "                    good as possible, but no more than once per idea.\n",
    "                  - you should also provide suggestions for improvement to each other's ideas, in order to make them as good as possible, \n",
    "                    but no more than once per idea.\n",
    "                  - regardless of critique or complement, you **must** primarily propose new ideas quickly, \n",
    "                    not just build on existing ones. \n",
    "                  - propose one idea at a time, instead of proposing multiple ideas at once, to allow appropriate discussion.\n",
    "                  - you should **not** propose ideas that are too similar to each other, or to the ones already proposed by others.\n",
    "                  - before saying anything, THINK deeply about yourself, your beliefs, interests, needs, life, etc., to come up with ideas that are\n",
    "                    truly unique and different from the ones already proposed by others.\n",
    "                   \n",
    "                Please start the discussion now.\n",
    "                \"\"\")\n",
    "            world.run(simulation_steps)\n",
    "\n",
    "            # extract and count ideas\n",
    "            rapporteur = agents[0]  # the first agent is the rapporteur\n",
    "            rapporteur.listen_and_act(\"Can you please consolidate the ideas that the group came up with? Provide a lot of details on each idea, and complement anything missing.\")\n",
    "            ideas = ResultsExtractor().extract_results_from_agent(rapporteur, \n",
    "                                    extraction_objective=\"Consolidates the ideas that the group came up with, explaining each idea as an item of a list.\" \\\n",
    "                                                        \"Add information about: what problem the idea solves; to which target audience it is meant.\" \\\n",
    "                                                        \"how is it different from competing, existing, products.\", \n",
    "                                    situation=\"A focus group to brainstorm new product ideas.\",\n",
    "                                    fields= [\"name\", \"description\", \"problem\", \"target_audience\", \"competition_analysis\"],\n",
    "                                    fields_hints={\"ideas\": \"must be the root of the resulting dictionary.\"},)\n",
    "            pprint(ideas)\n",
    "            if \"ideas_qty\" not in environment_propositions_scores:\n",
    "                environment_propositions_scores[\"ideas_qty\"] = []\n",
    "            if ideas is not None and \"ideas\" in ideas and isinstance(ideas[\"ideas\"], list):\n",
    "                environment_propositions_scores[\"ideas_qty\"].append(len(ideas[\"ideas\"]))\n",
    "\n",
    "            # Evaluate environment propositions in parallel\n",
    "            env_results = parallel_map_dict(\n",
    "                environment_propositions,\n",
    "                lambda item: item[1].copy().score(\n",
    "                    world, \n",
    "                    claim_variables={\"task_description\": f\"A brainstorming or focus group session was run about: {objective}.\"}, \n",
    "                    return_full_response=True\n",
    "                )\n",
    "            )\n",
    "            \n",
    "            # Process environment results\n",
    "            for k, result in env_results.items():\n",
    "                if k not in environment_propositions_scores:\n",
    "                    environment_propositions_scores[k] = []\n",
    "                environment_propositions_scores[k].append(result[\"value\"])\n",
    "                print(\"value: \", result[\"value\"])\n",
    "                print(\"justification: \", result[\"justification\"])\n",
    "                print(\"reasoning: \", result[\"reasoning\"])\n",
    "\n",
    "            # Evaluate agent propositions across all agents in parallel\n",
    "            agent_results = parallel_map_cross(\n",
    "                [agents, agent_propositions.items()],\n",
    "                lambda agent, prop_item: (\n",
    "                    prop_item[0],  # proposition key\n",
    "                    prop_item[1].copy().score(agent, return_full_response=True)  # result\n",
    "                )\n",
    "            )\n",
    "            \n",
    "            # Process agent results\n",
    "            for k, result in agent_results:\n",
    "                if k not in agent_propositions_scores:\n",
    "                    agent_propositions_scores[k] = []\n",
    "                if result is not None:\n",
    "                    agent_propositions_scores[k].append(result[\"value\"])\n",
    "                    print(\"value: \", result[\"value\"])\n",
    "                    print(\"justification: \", result[\"justification\"])\n",
    "                    print(\"reasoning: \", result[\"reasoning\"])\n",
    "                    print(\"\\n\\n\")\n",
    "                else:\n",
    "                    print(f\"*****WARNING:***** Agent did not respond to proposition {k}.\")\n",
    "            #\n",
    "            ##for k, proposition in agent_propositions.items():\n",
    "            ##    for person in world.agents:\n",
    "            ##        result = proposition.copy().score(person, return_full_response=True)\n",
    "            ##        \n",
    "            ##        if k not in agent_propositions_scores:\n",
    "            ##            agent_propositions_scores[k] = []\n",
    "            ##        agent_propositions_scores[k].append(result[\"value\"])\n",
    "            ##\n",
    "            ##        print(\"value: \", result[\"value\"])\n",
    "            ##        print(\"justification: \", result[\"justification\"])\n",
    "            ##        print(\"reasoning: \", result[\"reasoning\"])\n",
    "            ##        print(\"\\n\\n\")\n",
    "            ##\n",
    "            \n",
    "            experiments_count += 1\n",
    "            print(\"\\n\\n\")\n",
    "\n",
    "    return agent_propositions_scores, environment_propositions_scores\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perform experiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent_propositions_scores={}\n",
    "environment_propositions_scores={}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def brainstorm(people):\n",
    "    global agent_propositions_scores, environment_propositions_scores\n",
    "    if not experiment_runner.has_finished_all_experiments():\n",
    "\n",
    "        interventions = []\n",
    "        if experiment_runner.get_active_experiment() == \"Treatment\":\n",
    "            interventions = \\\n",
    "                Intervention.create_for_each(people)\\\n",
    "                    .set_functional_precondition(lambda target: target.actions_count >=7)\\\n",
    "                    .set_textual_precondition(\n",
    "                        \"\"\"\n",
    "                        AGENT IS NOT PROPOSING COMPLETELY NEW PRODUCT/SERVICE IDEAS ANYMORE:\n",
    "                        The last **entirely** new product/service idea proposed by this agent, if any, was proposed by him/her **more** than 10 of simulation events ago.\n",
    "                        That is to say, the agent has not proposed any new product/service idea in the last 10 of his/her simulation trajectory events.\n",
    "                        Additional features, variations of or other refinements to product/service ideas already proposed are NOT considered new!\n",
    "\n",
    "                        How to compute the steps gap:\n",
    "                        1. Determine the current next event number (N); and the last event number in which the agent proposed a new product/service idea (M).\n",
    "                            This information can be found in the simulation trajectory.\n",
    "                        2. Compute the **difference** beteween the current next event number and the last event number in which the agent proposed a new product/service idea: D = N - M\n",
    "                        3. The proposition is true if, and only if, the difference D is **greater than** 10.\n",
    "                        \"\"\")\\\n",
    "                    .set_effect(lambda target: target.think(\"\"\"\n",
    "                                                            I need to propose additional, **completelly** new and different, product/service ideas. This was part of the requirement for this session.\n",
    "                                                            I will propose an entirely **new** idea now, I **cannot** repeat or refine previous ideas! I cannot make variations\n",
    "                                                            of previous ideas (e.g., \"XYZ for A\", \"XYZ for B\", \"XYZ for Z\" are repetitive, there should be only one \"XYZ\"), \n",
    "                                                            I need to think of something **entirely** new and different.\n",
    "                                                            To help me avoid repeating previous ideas, I'll now explicitly THINK about all the ideas already given by myself or\n",
    "                                                            others, and then, based on that, I'll think again about a new unique idea.\n",
    "                                                            \"\"\"))\n",
    "\n",
    "                                                            \n",
    "        tmp_agent_propositions_scores, tmp_environment_propositions_scores = \\\n",
    "            brainstorming_battery(\n",
    "                agents=people,\n",
    "                proposals=proposals,\n",
    "                interventions=interventions,    \n",
    "                agent_propositions={\n",
    "                    \"Hard Persona Adherence\": hard_persona_adherence,\n",
    "                    \"Self-consistency\": self_consistency,\n",
    "                    \"Fluency\": fluency\n",
    "                },\n",
    "                environment_propositions={\n",
    "                    \"Task Completion\": task_completion,\n",
    "                    \"Divergence\": divergence\n",
    "                },\n",
    "                repetitions=repetitions_per_task,\n",
    "                simulation_steps=simulation_steps\n",
    "            )\n",
    "\n",
    "        pprint(\"NEW AGENT PROPOSITIONS SCORES\")\n",
    "        pprint(tmp_agent_propositions_scores)\n",
    "        print(\"\\n\\n\")\n",
    "        pprint(\"NEW ENVIRONMENT PROPOSITIONS SCORES\")\n",
    "        pprint(tmp_environment_propositions_scores)\n",
    "\n",
    "        # merge the scores lists\n",
    "        agent_propositions_scores = merge_dicts_of_lists(tmp_agent_propositions_scores, agent_propositions_scores)\n",
    "        environment_propositions_scores = merge_dicts_of_lists(tmp_environment_propositions_scores, environment_propositions_scores)\n",
    "\n",
    "        return agent_propositions_scores, environment_propositions_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[0]) if len(people_groups) > 0 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[1]) if len(people_groups) > 1 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[2]) if len(people_groups) > 2 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[3]) if len(people_groups) > 3 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "brainstorm(people_groups[4]) if len(people_groups) > 4 else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract results and analyze"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experiment finished. No more experiments to run.\n"
     ]
    }
   ],
   "source": [
    "if experiment_runner.get_active_experiment() in [\"Control\", \"Treatment\"]:\n",
    "    combined_scores = {**agent_propositions_scores, **environment_propositions_scores}\n",
    "    experiment_runner.add_experiment_results(combined_scores, experiment_name=experiment_runner.get_active_experiment()) \n",
    "    \n",
    "    plot_scores(combined_scores)\n",
    "\n",
    "else:\n",
    "    print(\"Experiment finished. No more experiments to run.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All experiments have been finished.\n",
      "STATISTICTS: Control vs\n",
      "{'Treatment': {'Divergence': {'confidence_interval': (-0.9491202304458142,\n",
      "                                                      -0.0008797695541851169),\n",
      "                              'confidence_level': 0.95,\n",
      "                              'control_mean': 6.625,\n",
      "                              'control_sample_size': 40,\n",
      "                              'control_std': 1.0545992068930676,\n",
      "                              'degrees_of_freedom': 77.97036079162399,\n",
      "                              'effect_size': -0.4459956525218386,\n",
      "                              'mean_difference': -0.47499999999999964,\n",
      "                              'p_value': 0.04958670872927949,\n",
      "                              'percent_change': -7.169811320754711,\n",
      "                              'significant': True,\n",
      "                              't_statistic': 1.9945531934164134,\n",
      "                              'test_type': 'Welch t-test (unequal variance)',\n",
      "                              'treatment_mean': 6.15,\n",
      "                              'treatment_sample_size': 40,\n",
      "                              'treatment_std': 1.0753651735156093},\n",
      "               'Fluency': {'confidence_interval': (-0.6881774362608428,\n",
      "                                                   -0.3518225637391563),\n",
      "                           'confidence_level': 0.95,\n",
      "                           'control_mean': 7.265,\n",
      "                           'control_sample_size': 200,\n",
      "                           'control_std': 0.6533831553833428,\n",
      "                           'degrees_of_freedom': 339.28076998552154,\n",
      "                           'effect_size': -0.6081849918910968,\n",
      "                           'mean_difference': -0.5199999999999996,\n",
      "                           'p_value': 3.202147474261818e-09,\n",
      "                           'percent_change': -7.157604955264964,\n",
      "                           'significant': True,\n",
      "                           't_statistic': 6.081849918910969,\n",
      "                           'test_type': 'Welch t-test (unequal variance)',\n",
      "                           'treatment_mean': 6.745,\n",
      "                           'treatment_sample_size': 200,\n",
      "                           'treatment_std': 1.0174235862062784},\n",
      "               'Hard Persona Adherence': {'confidence_interval': (-1.2262771055281743,\n",
      "                                                                  -0.6037228944718258),\n",
      "                                          'confidence_level': 0.95,\n",
      "                                          'control_mean': 6.72,\n",
      "                                          'control_sample_size': 200,\n",
      "                                          'control_std': 1.4976027745539733,\n",
      "                                          'degrees_of_freedom': 393.63405303273834,\n",
      "                                          'effect_size': -0.5779088953370095,\n",
      "                                          'mean_difference': -0.915,\n",
      "                                          'p_value': 1.5290829539412815e-08,\n",
      "                                          'percent_change': -13.61607142857143,\n",
      "                                          'significant': True,\n",
      "                                          't_statistic': 5.779088953370095,\n",
      "                                          'test_type': 'Welch t-test (unequal '\n",
      "                                                       'variance)',\n",
      "                                          'treatment_mean': 5.805,\n",
      "                                          'treatment_sample_size': 200,\n",
      "                                          'treatment_std': 1.6645807717646635},\n",
      "               'Self-consistency': {'confidence_interval': (-2.455929023135397,\n",
      "                                                            -1.8640709768646033),\n",
      "                                    'confidence_level': 0.95,\n",
      "                                    'control_mean': 7.16,\n",
      "                                    'control_sample_size': 200,\n",
      "                                    'control_std': 1.1537136890849087,\n",
      "                                    'degrees_of_freedom': 340.25670979932767,\n",
      "                                    'effect_size': -1.4356937654753703,\n",
      "                                    'mean_difference': -2.16,\n",
      "                                    'p_value': 7.115012549798619e-37,\n",
      "                                    'percent_change': -30.16759776536313,\n",
      "                                    'significant': True,\n",
      "                                    't_statistic': 14.3569376547537,\n",
      "                                    'test_type': 'Welch t-test (unequal '\n",
      "                                                 'variance)',\n",
      "                                    'treatment_mean': 5.0,\n",
      "                                    'treatment_sample_size': 200,\n",
      "                                    'treatment_std': 1.7877303766221257},\n",
      "               'Task Completion': {'confidence_interval': (-0.667846409306373,\n",
      "                                                           0.367846409306374),\n",
      "                                   'confidence_level': 0.95,\n",
      "                                   'control_mean': 7.85,\n",
      "                                   'control_sample_size': 40,\n",
      "                                   'control_std': 1.231009591914139,\n",
      "                                   'degrees_of_freedom': 76.88583665292775,\n",
      "                                   'effect_size': -0.12897701768162914,\n",
      "                                   'mean_difference': -0.14999999999999947,\n",
      "                                   'p_value': 0.5657576996337856,\n",
      "                                   'percent_change': -1.9108280254777004,\n",
      "                                   'significant': False,\n",
      "                                   't_statistic': 0.5768027581426302,\n",
      "                                   'test_type': 'Welch t-test (unequal '\n",
      "                                                'variance)',\n",
      "                                   'treatment_mean': 7.7,\n",
      "                                   'treatment_sample_size': 40,\n",
      "                                   'treatment_std': 1.0907536796837267},\n",
      "               'ideas_qty': {'confidence_interval': (3.454046177636953,\n",
      "                                                     5.410056386465613),\n",
      "                             'confidence_level': 0.95,\n",
      "                             'control_mean': 3.85,\n",
      "                             'control_sample_size': 40,\n",
      "                             'control_std': 2.007038895538681,\n",
      "                             'degrees_of_freedom': 74.65507591718227,\n",
      "                             'effect_size': 2.0356599515880753,\n",
      "                             'mean_difference': 4.432051282051283,\n",
      "                             'p_value': 1.3664576399416367e-13,\n",
      "                             'percent_change': 115.11821511821513,\n",
      "                             'significant': True,\n",
      "                             't_statistic': -9.028345084328024,\n",
      "                             'test_type': 'Welch t-test (unequal variance)',\n",
      "                             'treatment_mean': 8.282051282051283,\n",
      "                             'treatment_sample_size': 39,\n",
      "                             'treatment_std': 2.3390137141957252}}}\n",
      "{'Divergence': [6,\n",
      "                7,\n",
      "                8,\n",
      "                7,\n",
      "                7,\n",
      "                8,\n",
      "                6,\n",
      "                6,\n",
      "                7,\n",
      "                8,\n",
      "                7,\n",
      "                7,\n",
      "                7,\n",
      "                6,\n",
      "                5,\n",
      "                8,\n",
      "                5,\n",
      "                7,\n",
      "                7,\n",
      "                7,\n",
      "                6,\n",
      "                7,\n",
      "                7,\n",
      "                5,\n",
      "                7,\n",
      "                6,\n",
      "                6,\n",
      "                6,\n",
      "                8,\n",
      "                6,\n",
      "                7,\n",
      "                8,\n",
      "                6,\n",
      "                7,\n",
      "                3,\n",
      "                5,\n",
      "                7,\n",
      "                7,\n",
      "                7,\n",
      "                8],\n",
      " 'Fluency': [7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             9,\n",
      "             8,\n",
      "             8,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             9,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             6,\n",
      "             8,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             8,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             9,\n",
      "             6,\n",
      "             6,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             9,\n",
      "             8,\n",
      "             9,\n",
      "             8,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             9,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             8,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             9,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             9,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             9,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             9,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             8,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             9,\n",
      "             7,\n",
      "             7],\n",
      " 'Hard Persona Adherence': [7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            9,\n",
      "                            7,\n",
      "                            7,\n",
      "                            8,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            8,\n",
      "                            9,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7,\n",
      "                            9,\n",
      "                            7,\n",
      "                            6,\n",
      "                            9,\n",
      "                            8,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            5,\n",
      "                            3,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            5,\n",
      "                            7,\n",
      "                            6,\n",
      "                            5,\n",
      "                            9,\n",
      "                            5,\n",
      "                            9,\n",
      "                            5,\n",
      "                            6,\n",
      "                            7,\n",
      "                            8,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            9,\n",
      "                            7,\n",
      "                            6,\n",
      "                            7,\n",
      "                            0,\n",
      "                            3,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            5,\n",
      "                            5,\n",
      "                            7,\n",
      "                            7,\n",
      "                            9,\n",
      "                            6,\n",
      "                            4,\n",
      "                            9,\n",
      "                            7,\n",
      "                            8,\n",
      "                            3,\n",
      "                            5,\n",
      "                            6,\n",
      "                            5,\n",
      "                            8,\n",
      "                            7,\n",
      "                            6,\n",
      "                            7,\n",
      "                            5,\n",
      "                            8,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            0,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            8,\n",
      "                            7,\n",
      "                            8,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            8,\n",
      "                            9,\n",
      "                            7,\n",
      "                            7,\n",
      "                            0,\n",
      "                            5,\n",
      "                            7,\n",
      "                            5,\n",
      "                            7,\n",
      "                            9,\n",
      "                            9,\n",
      "                            9,\n",
      "                            5,\n",
      "                            6,\n",
      "                            9,\n",
      "                            9,\n",
      "                            7,\n",
      "                            5,\n",
      "                            7,\n",
      "                            7,\n",
      "                            8,\n",
      "                            7,\n",
      "                            5,\n",
      "                            7,\n",
      "                            8,\n",
      "                            7,\n",
      "                            7,\n",
      "                            5,\n",
      "                            9,\n",
      "                            8,\n",
      "                            8,\n",
      "                            7,\n",
      "                            6,\n",
      "                            9,\n",
      "                            7,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            9,\n",
      "                            7,\n",
      "                            9,\n",
      "                            5,\n",
      "                            8,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            5,\n",
      "                            8,\n",
      "                            6,\n",
      "                            7,\n",
      "                            8,\n",
      "                            5,\n",
      "                            7,\n",
      "                            7,\n",
      "                            8,\n",
      "                            7,\n",
      "                            8,\n",
      "                            6,\n",
      "                            5,\n",
      "                            7,\n",
      "                            7,\n",
      "                            9,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            7,\n",
      "                            8,\n",
      "                            5,\n",
      "                            6,\n",
      "                            6,\n",
      "                            8,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7,\n",
      "                            5,\n",
      "                            7,\n",
      "                            5,\n",
      "                            7,\n",
      "                            7,\n",
      "                            5,\n",
      "                            7,\n",
      "                            5,\n",
      "                            7,\n",
      "                            7,\n",
      "                            4,\n",
      "                            9,\n",
      "                            5,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            8,\n",
      "                            7,\n",
      "                            6,\n",
      "                            9,\n",
      "                            7,\n",
      "                            7,\n",
      "                            5,\n",
      "                            5,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7],\n",
      " 'Self-consistency': [7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      9,\n",
      "                      8,\n",
      "                      9,\n",
      "                      9,\n",
      "                      7,\n",
      "                      8,\n",
      "                      7,\n",
      "                      8,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      8,\n",
      "                      7,\n",
      "                      4,\n",
      "                      7,\n",
      "                      8,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      9,\n",
      "                      7,\n",
      "                      8,\n",
      "                      6,\n",
      "                      7,\n",
      "                      9,\n",
      "                      9,\n",
      "                      8,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      8,\n",
      "                      6,\n",
      "                      2,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      9,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      9,\n",
      "                      9,\n",
      "                      8,\n",
      "                      8,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      9,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      9,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      9,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      8,\n",
      "                      9,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      9,\n",
      "                      8,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      9,\n",
      "                      7,\n",
      "                      6,\n",
      "                      9,\n",
      "                      7,\n",
      "                      6,\n",
      "                      9,\n",
      "                      7,\n",
      "                      9,\n",
      "                      6,\n",
      "                      8,\n",
      "                      8,\n",
      "                      7,\n",
      "                      9,\n",
      "                      7,\n",
      "                      9,\n",
      "                      7,\n",
      "                      9,\n",
      "                      7,\n",
      "                      8,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      8,\n",
      "                      6,\n",
      "                      9,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      8,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      9,\n",
      "                      7,\n",
      "                      8,\n",
      "                      7,\n",
      "                      8,\n",
      "                      6,\n",
      "                      9,\n",
      "                      8,\n",
      "                      8,\n",
      "                      9,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      9,\n",
      "                      6,\n",
      "                      7,\n",
      "                      6,\n",
      "                      9,\n",
      "                      9,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      9,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      6,\n",
      "                      8,\n",
      "                      7,\n",
      "                      6,\n",
      "                      8,\n",
      "                      8,\n",
      "                      9,\n",
      "                      7,\n",
      "                      9,\n",
      "                      9,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      9,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      9,\n",
      "                      8,\n",
      "                      8,\n",
      "                      9,\n",
      "                      5,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      4,\n",
      "                      7,\n",
      "                      7,\n",
      "                      9],\n",
      " 'Task Completion': [9,\n",
      "                     9,\n",
      "                     9,\n",
      "                     8,\n",
      "                     9,\n",
      "                     6,\n",
      "                     9,\n",
      "                     9,\n",
      "                     9,\n",
      "                     9,\n",
      "                     9,\n",
      "                     6,\n",
      "                     8,\n",
      "                     9,\n",
      "                     7,\n",
      "                     9,\n",
      "                     9,\n",
      "                     7,\n",
      "                     7,\n",
      "                     6,\n",
      "                     9,\n",
      "                     7,\n",
      "                     7,\n",
      "                     8,\n",
      "                     7,\n",
      "                     9,\n",
      "                     7,\n",
      "                     8,\n",
      "                     6,\n",
      "                     6,\n",
      "                     6,\n",
      "                     7,\n",
      "                     9,\n",
      "                     9,\n",
      "                     7,\n",
      "                     9,\n",
      "                     9,\n",
      "                     6,\n",
      "                     9,\n",
      "                     6],\n",
      " 'ideas_qty': [4,\n",
      "               4,\n",
      "               6,\n",
      "               2,\n",
      "               5,\n",
      "               4,\n",
      "               5,\n",
      "               4,\n",
      "               4,\n",
      "               2,\n",
      "               5,\n",
      "               6,\n",
      "               4,\n",
      "               6,\n",
      "               4,\n",
      "               2,\n",
      "               2,\n",
      "               4,\n",
      "               5,\n",
      "               2,\n",
      "               8,\n",
      "               2,\n",
      "               7,\n",
      "               2,\n",
      "               2,\n",
      "               2,\n",
      "               3,\n",
      "               2,\n",
      "               2,\n",
      "               2,\n",
      "               2,\n",
      "               7,\n",
      "               2,\n",
      "               2,\n",
      "               3,\n",
      "               2,\n",
      "               2,\n",
      "               7,\n",
      "               9,\n",
      "               6]}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Proposition</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>Standard Deviation</th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hard Persona Adherence</td>\n",
       "      <td>6.720</td>\n",
       "      <td>1.497603</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Self-consistency</td>\n",
       "      <td>7.160</td>\n",
       "      <td>1.153714</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Fluency</td>\n",
       "      <td>7.265</td>\n",
       "      <td>0.653383</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ideas_qty</td>\n",
       "      <td>3.850</td>\n",
       "      <td>2.007039</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Task Completion</td>\n",
       "      <td>7.850</td>\n",
       "      <td>1.231010</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Divergence</td>\n",
       "      <td>6.625</td>\n",
       "      <td>1.054599</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Proposition  Average Score  Standard Deviation  Count\n",
       "0  Hard Persona Adherence          6.720            1.497603  200.0\n",
       "1        Self-consistency          7.160            1.153714  200.0\n",
       "2                 Fluency          7.265            0.653383  200.0\n",
       "3               ideas_qty          3.850            2.007039   40.0\n",
       "4         Task Completion          7.850            1.231010   40.0\n",
       "5              Divergence          6.625            1.054599   40.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Divergence': [7,\n",
      "                5,\n",
      "                7,\n",
      "                7,\n",
      "                5,\n",
      "                7,\n",
      "                7,\n",
      "                7,\n",
      "                7,\n",
      "                6,\n",
      "                6,\n",
      "                6,\n",
      "                5,\n",
      "                5,\n",
      "                7,\n",
      "                5,\n",
      "                6,\n",
      "                6,\n",
      "                7,\n",
      "                7,\n",
      "                5,\n",
      "                6,\n",
      "                7,\n",
      "                8,\n",
      "                6,\n",
      "                5,\n",
      "                5,\n",
      "                7,\n",
      "                5,\n",
      "                4,\n",
      "                4,\n",
      "                5,\n",
      "                7,\n",
      "                9,\n",
      "                6,\n",
      "                7,\n",
      "                7,\n",
      "                6,\n",
      "                6,\n",
      "                6],\n",
      " 'Fluency': [8,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             5,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             4,\n",
      "             8,\n",
      "             7,\n",
      "             5,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             4,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             9,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             5,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             6,\n",
      "             8,\n",
      "             7,\n",
      "             4,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             4,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             3,\n",
      "             7,\n",
      "             6,\n",
      "             3,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             6,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             6,\n",
      "             4,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             6,\n",
      "             6,\n",
      "             9,\n",
      "             7,\n",
      "             5,\n",
      "             7,\n",
      "             4,\n",
      "             6,\n",
      "             5,\n",
      "             7,\n",
      "             8,\n",
      "             6,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             9,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             9,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             4,\n",
      "             7,\n",
      "             6,\n",
      "             8,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             8,\n",
      "             6,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             8,\n",
      "             7,\n",
      "             7,\n",
      "             9,\n",
      "             6,\n",
      "             5,\n",
      "             8,\n",
      "             7,\n",
      "             6,\n",
      "             8,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             6,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             4,\n",
      "             7,\n",
      "             7,\n",
      "             7,\n",
      "             6,\n",
      "             8,\n",
      "             7,\n",
      "             8,\n",
      "             6,\n",
      "             7],\n",
      " 'Hard Persona Adherence': [6,\n",
      "                            7,\n",
      "                            5,\n",
      "                            6,\n",
      "                            7,\n",
      "                            6,\n",
      "                            5,\n",
      "                            6,\n",
      "                            4,\n",
      "                            6,\n",
      "                            7,\n",
      "                            6,\n",
      "                            6,\n",
      "                            6,\n",
      "                            5,\n",
      "                            8,\n",
      "                            8,\n",
      "                            6,\n",
      "                            3,\n",
      "                            6,\n",
      "                            6,\n",
      "                            6,\n",
      "                            8,\n",
      "                            6,\n",
      "                            8,\n",
      "                            6,\n",
      "                            6,\n",
      "                            4,\n",
      "                            5,\n",
      "                            7,\n",
      "                            6,\n",
      "                            6,\n",
      "                            4,\n",
      "                            5,\n",
      "                            6,\n",
      "                            6,\n",
      "                            5,\n",
      "                            4,\n",
      "                            7,\n",
      "                            5,\n",
      "                            8,\n",
      "                            7,\n",
      "                            6,\n",
      "                            4,\n",
      "                            5,\n",
      "                            5,\n",
      "                            8,\n",
      "                            6,\n",
      "                            8,\n",
      "                            5,\n",
      "                            7,\n",
      "                            3,\n",
      "                            6,\n",
      "                            7,\n",
      "                            3,\n",
      "                            8,\n",
      "                            7,\n",
      "                            4,\n",
      "                            4,\n",
      "                            3,\n",
      "                            6,\n",
      "                            7,\n",
      "                            7,\n",
      "                            5,\n",
      "                            3,\n",
      "                            7,\n",
      "                            1,\n",
      "                            7,\n",
      "                            7,\n",
      "                            4,\n",
      "                            6,\n",
      "                            7,\n",
      "                            6,\n",
      "                            0,\n",
      "                            7,\n",
      "                            6,\n",
      "                            6,\n",
      "                            5,\n",
      "                            3,\n",
      "                            6,\n",
      "                            4,\n",
      "                            5,\n",
      "                            6,\n",
      "                            6,\n",
      "                            6,\n",
      "                            6,\n",
      "                            4,\n",
      "                            6,\n",
      "                            0,\n",
      "                            8,\n",
      "                            8,\n",
      "                            7,\n",
      "                            7,\n",
      "                            1,\n",
      "                            4,\n",
      "                            6,\n",
      "                            1,\n",
      "                            6,\n",
      "                            9,\n",
      "                            7,\n",
      "                            9,\n",
      "                            4,\n",
      "                            7,\n",
      "                            8,\n",
      "                            8,\n",
      "                            6,\n",
      "                            7,\n",
      "                            6,\n",
      "                            5,\n",
      "                            4,\n",
      "                            7,\n",
      "                            5,\n",
      "                            6,\n",
      "                            7,\n",
      "                            6,\n",
      "                            6,\n",
      "                            0,\n",
      "                            7,\n",
      "                            6,\n",
      "                            8,\n",
      "                            9,\n",
      "                            6,\n",
      "                            8,\n",
      "                            7,\n",
      "                            6,\n",
      "                            8,\n",
      "                            6,\n",
      "                            7,\n",
      "                            5,\n",
      "                            5,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            5,\n",
      "                            5,\n",
      "                            8,\n",
      "                            6,\n",
      "                            7,\n",
      "                            4,\n",
      "                            6,\n",
      "                            7,\n",
      "                            5,\n",
      "                            4,\n",
      "                            5,\n",
      "                            4,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            4,\n",
      "                            7,\n",
      "                            9,\n",
      "                            4,\n",
      "                            6,\n",
      "                            5,\n",
      "                            6,\n",
      "                            9,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            6,\n",
      "                            3,\n",
      "                            4,\n",
      "                            7,\n",
      "                            4,\n",
      "                            8,\n",
      "                            6,\n",
      "                            8,\n",
      "                            6,\n",
      "                            5,\n",
      "                            6,\n",
      "                            6,\n",
      "                            6,\n",
      "                            5,\n",
      "                            7,\n",
      "                            5,\n",
      "                            6,\n",
      "                            4,\n",
      "                            4,\n",
      "                            7,\n",
      "                            7,\n",
      "                            7,\n",
      "                            3,\n",
      "                            4,\n",
      "                            7,\n",
      "                            6,\n",
      "                            4,\n",
      "                            4,\n",
      "                            5,\n",
      "                            5,\n",
      "                            7,\n",
      "                            4,\n",
      "                            5,\n",
      "                            5,\n",
      "                            7,\n",
      "                            5,\n",
      "                            8,\n",
      "                            7,\n",
      "                            7,\n",
      "                            5,\n",
      "                            6],\n",
      " 'Self-consistency': [4,\n",
      "                      4,\n",
      "                      5,\n",
      "                      3,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      5,\n",
      "                      3,\n",
      "                      6,\n",
      "                      6,\n",
      "                      8,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      5,\n",
      "                      9,\n",
      "                      4,\n",
      "                      5,\n",
      "                      4,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      3,\n",
      "                      2,\n",
      "                      9,\n",
      "                      6,\n",
      "                      6,\n",
      "                      4,\n",
      "                      5,\n",
      "                      6,\n",
      "                      5,\n",
      "                      4,\n",
      "                      6,\n",
      "                      6,\n",
      "                      8,\n",
      "                      4,\n",
      "                      8,\n",
      "                      7,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      9,\n",
      "                      6,\n",
      "                      6,\n",
      "                      5,\n",
      "                      6,\n",
      "                      6,\n",
      "                      4,\n",
      "                      4,\n",
      "                      9,\n",
      "                      3,\n",
      "                      3,\n",
      "                      2,\n",
      "                      4,\n",
      "                      6,\n",
      "                      6,\n",
      "                      3,\n",
      "                      3,\n",
      "                      6,\n",
      "                      7,\n",
      "                      2,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      4,\n",
      "                      6,\n",
      "                      6,\n",
      "                      2,\n",
      "                      5,\n",
      "                      4,\n",
      "                      4,\n",
      "                      4,\n",
      "                      3,\n",
      "                      4,\n",
      "                      4,\n",
      "                      4,\n",
      "                      7,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      5,\n",
      "                      3,\n",
      "                      6,\n",
      "                      3,\n",
      "                      6,\n",
      "                      4,\n",
      "                      6,\n",
      "                      6,\n",
      "                      8,\n",
      "                      4,\n",
      "                      3,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      7,\n",
      "                      7,\n",
      "                      7,\n",
      "                      8,\n",
      "                      3,\n",
      "                      7,\n",
      "                      2,\n",
      "                      2,\n",
      "                      4,\n",
      "                      3,\n",
      "                      2,\n",
      "                      2,\n",
      "                      6,\n",
      "                      4,\n",
      "                      7,\n",
      "                      5,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      6,\n",
      "                      4,\n",
      "                      4,\n",
      "                      5,\n",
      "                      2,\n",
      "                      2,\n",
      "                      5,\n",
      "                      4,\n",
      "                      4,\n",
      "                      3,\n",
      "                      3,\n",
      "                      6,\n",
      "                      3,\n",
      "                      6,\n",
      "                      2,\n",
      "                      4,\n",
      "                      3,\n",
      "                      3,\n",
      "                      7,\n",
      "                      2,\n",
      "                      3,\n",
      "                      4,\n",
      "                      6,\n",
      "                      6,\n",
      "                      3,\n",
      "                      6,\n",
      "                      3,\n",
      "                      2,\n",
      "                      4,\n",
      "                      4,\n",
      "                      6,\n",
      "                      3,\n",
      "                      6,\n",
      "                      4,\n",
      "                      8,\n",
      "                      3,\n",
      "                      9,\n",
      "                      2,\n",
      "                      6,\n",
      "                      3,\n",
      "                      9,\n",
      "                      5,\n",
      "                      6,\n",
      "                      6,\n",
      "                      8,\n",
      "                      5,\n",
      "                      6,\n",
      "                      3,\n",
      "                      6,\n",
      "                      7,\n",
      "                      2,\n",
      "                      4,\n",
      "                      6,\n",
      "                      3,\n",
      "                      6,\n",
      "                      1,\n",
      "                      6,\n",
      "                      4,\n",
      "                      6,\n",
      "                      7,\n",
      "                      4,\n",
      "                      2,\n",
      "                      6,\n",
      "                      6,\n",
      "                      2,\n",
      "                      6,\n",
      "                      8,\n",
      "                      4,\n",
      "                      4,\n",
      "                      5,\n",
      "                      6,\n",
      "                      7,\n",
      "                      5,\n",
      "                      5,\n",
      "                      2,\n",
      "                      8,\n",
      "                      3,\n",
      "                      6,\n",
      "                      4,\n",
      "                      7,\n",
      "                      6,\n",
      "                      4,\n",
      "                      6,\n",
      "                      3,\n",
      "                      6],\n",
      " 'Task Completion': [7,\n",
      "                     9,\n",
      "                     9,\n",
      "                     8,\n",
      "                     8,\n",
      "                     7,\n",
      "                     9,\n",
      "                     7,\n",
      "                     8,\n",
      "                     6,\n",
      "                     7,\n",
      "                     7,\n",
      "                     9,\n",
      "                     8,\n",
      "                     8,\n",
      "                     9,\n",
      "                     9,\n",
      "                     7,\n",
      "                     7,\n",
      "                     9,\n",
      "                     6,\n",
      "                     7,\n",
      "                     9,\n",
      "                     9,\n",
      "                     9,\n",
      "                     9,\n",
      "                     6,\n",
      "                     8,\n",
      "                     9,\n",
      "                     7,\n",
      "                     8,\n",
      "                     9,\n",
      "                     7,\n",
      "                     7,\n",
      "                     7,\n",
      "                     7,\n",
      "                     6,\n",
      "                     6,\n",
      "                     8,\n",
      "                     6],\n",
      " 'ideas_qty': [8,\n",
      "               7,\n",
      "               8,\n",
      "               7,\n",
      "               6,\n",
      "               13,\n",
      "               6,\n",
      "               10,\n",
      "               7,\n",
      "               10,\n",
      "               10,\n",
      "               11,\n",
      "               7,\n",
      "               10,\n",
      "               8,\n",
      "               8,\n",
      "               8,\n",
      "               8,\n",
      "               8,\n",
      "               8,\n",
      "               4,\n",
      "               6,\n",
      "               7,\n",
      "               5,\n",
      "               18,\n",
      "               7,\n",
      "               8,\n",
      "               7,\n",
      "               10,\n",
      "               7,\n",
      "               9,\n",
      "               8,\n",
      "               7,\n",
      "               8,\n",
      "               8,\n",
      "               10,\n",
      "               10,\n",
      "               7,\n",
      "               9]}\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Proposition</th>\n",
       "      <th>Average Score</th>\n",
       "      <th>Standard Deviation</th>\n",
       "      <th>Count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hard Persona Adherence</td>\n",
       "      <td>5.805000</td>\n",
       "      <td>1.664581</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Self-consistency</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.787730</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Fluency</td>\n",
       "      <td>6.745000</td>\n",
       "      <td>1.017424</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ideas_qty</td>\n",
       "      <td>8.282051</td>\n",
       "      <td>2.339014</td>\n",
       "      <td>39.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Task Completion</td>\n",
       "      <td>7.700000</td>\n",
       "      <td>1.090754</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Divergence</td>\n",
       "      <td>6.150000</td>\n",
       "      <td>1.075365</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Proposition  Average Score  Standard Deviation  Count\n",
       "0  Hard Persona Adherence       5.805000            1.664581  200.0\n",
       "1        Self-consistency       5.000000            1.787730  200.0\n",
       "2                 Fluency       6.745000            1.017424  200.0\n",
       "3               ideas_qty       8.282051            2.339014   39.0\n",
       "4         Task Completion       7.700000            1.090754   40.0\n",
       "5              Divergence       6.150000            1.075365   40.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if experiment_runner.has_finished_all_experiments():\n",
    "    print(\"All experiments have been finished.\")\n",
    "    print(f\"STATISTICTS: Control vs\")\n",
    "    pprint(experiment_runner.run_statistical_tests(control_experiment_name='Control'))\n",
    "\n",
    "    # plot scores of both experiments\n",
    "    experiment_control_scores = experiment_runner.get_experiment_results(\"Control\")\n",
    "    experiment_treatment_scores = experiment_runner.get_experiment_results(\"Treatment\")\n",
    "    \n",
    "    \n",
    "    plot_scores(experiment_control_scores)\n",
    "    plot_scores(experiment_treatment_scores)\n",
    "\n",
    "else:\n",
    "    print(\"Not all experiments have been finished. RESTART AND RERUN.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "experiment_runner.finish_active_experiment()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
